# -*- coding: utf-8 -*-
import scrapy,requests,json,demjson
from sinanews.sqlHelper import *

class NewsSpider(scrapy.Spider):
    name = 'news'
    allowed_domains = ['sina.com.cn']
    #采集深度
    spider_page=10
    type_index=0
    url='http://roll.news.sina.com.cn/interface/rollnews_ch_out_interface.php?col=89&spec=&type=&ch=01&k=&offset_page=0&offset_num=0&num=60&asc=&page='
    offset=0
    page_index=1
    url_list=[]
    print '采集类别：新闻'
    def parse_list(self,res):
        print '开始采集第'+str(self.page_index)+"页"
        res=res.body
        res=res[15:len(res)-1]
        res=res.decode("gb18030")
        res=demjson.decode(res)
        self.url_list=res['list']
        print '当前页共有['+str(len(self.url_list))+"]条数据"
        self.offset=0
        if self.page_index<self.spider_page:
            self.page_index+=1
        print '正在采集第'+str(self.page_index-1)+"页 第["+str(self.offset)+"]条数据"
        yield scrapy.Request(self.url_list[self.offset]['url'],callback=self.parse)
        

    def parse(self, response):
        if response.status==404:
            print '******404错误******'
        save=True
        for each in response.xpath('//*[@class="article"]/img'):
            url=each.xpath('./@src').extract()[0]
            ftype=url[len(url)-4:len(url)].lower()
            if ftype!='.jpg' and ftype!='png':
                save=False
        if save:
            try:
                content=response.xpath('//*[@class="article"]').extract()[0]
                title=response.xpath('//*[@class="main-title"]/text()').extract()[0]
                date=response.xpath('//*[@id="top_bar"]/div/div[2]/span/text()').extract()[0]
                category='新闻'
                if sqlHelper.selectOne("select title from news where title=%s",(title)):
                    print '重复内容，抛弃'
                else:
                    sqlHelper.update("insert into news(title,category,content,date) values(%s,%s,%s,%s)",(title,category,content,date))
            except:
                print '地址'+response.url+"采集异常"
        else:
            print '图片格式不匹配，抛弃！'

        if self.offset<len(self.url_list):
            self.offset+=1
            url=self.url_list[self.offset]['url']
            url=url.replace("\\","")
            print '正在采集第'+str(self.page_index-1)+"页 第["+str(self.offset)+"]条数据"
            yield scrapy.Request(url,callback=self.parse,dont_filter=True)
        else:
            yield scrapy.Request(self.url+str(self.page_index),self.parse_list)
    def start_requests(self):
        page=scrapy.Request(self.url+str(self.page_index),self.parse_list)
        return [page]
